package stackOverflow;

import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import stackOverflow.PostVotesContainer.Votes;

public class DataLoader {

    //Path to XMLs
    private String usersXML	
    	= "C:\\Stack Overflow Data Dump - Jun 2011\\Content\\askubuntu.stackexchange.com\\062011 Ask Ubuntu\\users.xml";
    private String postsXML	
    	= "C:\\Stack Overflow Data Dump - Jun 2011\\Content\\askubuntu.stackexchange.com\\062011 Ask Ubuntu\\posts.xml";
    private String votesXML	
    	= "C:\\Stack Overflow Data Dump - Jun 2011\\Content\\askubuntu.stackexchange.com\\062011 Ask Ubuntu\\votes.xml";
    
    private final static String ELEMENT_DATA_NAME = "row";
    
    /**
     * @author Boris Cherkasky
     * 
     * XML element attributes of post.xml
     */
    public enum PostAttributes{
	POST_TYPE	("PostTypeId"),
	ACCEPTED_ANS_ID	("AcceptedAnswerId"),
	ID		("Id"),
	SCORE		("Score"),
	VIEW_COUNT	("ViewCount"),
	AUTHOR_USER_ID	("OwnerUserId"),
	BODY		("Body"),
	CREATION_DATE	("CreationDate"),
	LAST_ACTIVITY	("LastActivityDate"),
	TITLE		("Title"),
	PARENT_ID	("ParentId");
	
	private final String field;	    
	PostAttributes(String _field){field = _field;}
	public String value(){return field;}
    }
    /**
     * @author Boris Cherkasky
     * 
     * XML element attributes of users.xml
     */
    public enum UserAttributes{
	REPUTATION	("Reputation"),
	DISPLAY_NAME	("DisplayName"),
	ID		("Id");
	
	private final String field;	    
	UserAttributes(String _field){field = _field;}
	public String value(){return field;}
    }
    
    /**
     * @author Boris Cherkasky
     * 
     * XML element attributes of votes.xml
     */
    public enum VotesAttributes{
	VOTE_TYPE	("VoteTypeId"),
	POST_ID		("PostId");
	
	//values of the VOTE_TYPE attribute
	public final static int UP_VOTE 	= 2;
	public final static int DOWN_VOTE	= 3;
	
	private final String field;	    
	VotesAttributes(String _field){field = _field;}
	public String value(){return field;}
	/**
	 * Check if the vote type is one of the two we are interested in (UP/DOWN votes)
	 * 
	 * @param type
	 * @return
	 */
	public boolean isKnownVoteType(int type){
	    return type == UP_VOTE || type == DOWN_VOTE;
	}
	
    }

    /**
     * Overwrite path to xmls with new values 
     * 
     * @param postsXML
     * @param userXML
     * @param votesXML
     */
    public DataLoader(String postsXML, String userXML, String votesXML){
	if (userXML != null){
	    this.usersXML = userXML;
	}
	if (postsXML != null){
	    this.postsXML = postsXML;
	}
	if (votesXML != null){
	    this.votesXML = votesXML;
	}
    }
    
    /**
     * Use path to xmls as defined in code 
     */
    public DataLoader() {}

    /**
     * Load the XML document 
     * @param path - path to the XML file
     * @return Document representation of the XML file 
     * @throws ParserConfigurationException 
     * @throws IOException 
     * @throws SAXException 
     */
    private Document getXML(String path) throws SAXException, IOException, ParserConfigurationException{
	File f = new File(path);
	Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(f);
	return doc;
    }
    
    /**
     * Get the {count} code objects from the XML, starting at {from}
     * @param doc
     * @param from - first code object to extract
     * @param amount - the amount of objects to extract. to extract ALL code obejcts - set amount = -1
     * @param loadOnlyJava TODO
     * @return
     * @throws SQLException 
     * @throws ClassNotFoundException 
     * @throws IllegalAccessException 
     * @throws InstantiationException 
     */
    private void loadPosts(Document doc, int from, int amount, boolean loadOnlyJava) 
    		throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException{
	DBhandler db = new DBhandler();
	ArrayList<Integer> acceptedAnswers = new ArrayList<Integer>();
	HashMap<Integer, Integer> ansToParentMap = new HashMap<Integer, Integer>(); //a map: answerID => questionID
	
	NodeList nodes = doc.getElementsByTagName(ELEMENT_DATA_NAME);
	amount = (amount == -1) ? nodes.getLength() : amount;
	System.out.println("-- Loading " + amount + " posts.");
	//iterate nodes and extract code object from each one
	for (int i = from ; i < from + amount ; i++){
	    int acceptedAnswerID = -1, postID =-1, score=0, viewCount=0, authorID=-1, parentID = -1;
	    boolean isQuestion;
	    Element e = (Element) nodes.item(i);
	    //extract code object data
	    
	    if (loadOnlyJava){
		//validate that question contains "java" tag
	    	String tags = e.getAttribute("Tags");
	    	if (tags == null || !tagExists(tags, "java")){
	    	    continue;
    	    	}
	    }
	    
	    try{
		//the only mandatory fields
		postID		= Integer.parseInt(e.getAttribute(PostAttributes.ID.value()));
		authorID	= Integer.parseInt(e.getAttribute(PostAttributes.AUTHOR_USER_ID.value()));
		
		//check if post is question or answer
		isQuestion = Integer.parseInt(e.getAttribute(PostAttributes.POST_TYPE.value())) == 1;
		if (isQuestion){
		    //if questions - get the accepted answer id
		    try{
    		    	acceptedAnswerID = Integer.parseInt(e.getAttribute(PostAttributes.ACCEPTED_ANS_ID.value()));
		    } catch (NumberFormatException e2) {
			//no accepted answer
    		    	acceptedAnswerID = -1;
    		    }
		} else {
		    //Get parentID to later set the title of answers
		    parentID = Integer.parseInt(e.getAttribute(PostAttributes.PARENT_ID.value()));
		    ansToParentMap.put(postID, parentID);
		}
		
		try{
		    score	= Integer.parseInt(e.getAttribute(PostAttributes.SCORE.value()));
		} catch (NumberFormatException e2) {score = -1;}
		try{
		    viewCount	= Integer.parseInt(e.getAttribute(PostAttributes.VIEW_COUNT.value()));
		} catch (NumberFormatException e2) {viewCount = -1;}
		
	    } catch (NumberFormatException e1){
		continue;
	    }
	    
	    String body 	 = e.getAttribute(PostAttributes.BODY.value());
	    String creationDate  = e.getAttribute(PostAttributes.CREATION_DATE.value());
	    String lastAcctivity = e.getAttribute(PostAttributes.LAST_ACTIVITY.value());
	    String title	 = e.getAttribute(PostAttributes.TITLE.value());
	    if (title == null || body == null || postID == -1 || authorID == -1){
		//Some attributes are missing - disregard post
		continue;
	    }
	    boolean isAccepted = acceptedAnswers.contains(postID);
	    if (isAccepted){
		//remove postID from acceptedAnswers - we found it! 
		acceptedAnswers.remove(new Integer(postID));
	    }
	    //Add post to the DB
	    db.addPost(creationDate, lastAcctivity, title, postID, score, 
		    viewCount, isAccepted, isQuestion, body, 0, 0, authorID);
	    if (acceptedAnswerID != -1){
		acceptedAnswers.add(acceptedAnswerID);
	    }
	}
	//Set accepted answers
	addAcceptedAnswers(acceptedAnswers, db);
	//Set answer titles
	addAnswerTitles(ansToParentMap, db, nodes);
	//commit everything
	db.executeBatches();
    }
    
    /**
     * Add titles question titles to all answers
     * 
     * @param ansToParentMap - a mapping: answerID => questionID
     * @param db
     * @param nodes TODO
     * @throws SQLException
     */
    private void addAnswerTitles(HashMap<Integer, Integer> ansToParentMap,
	    DBhandler db, NodeList nodes) throws SQLException {
	System.out.println("- Setting answer titles...");
	for (Entry<Integer, Integer> entry : ansToParentMap.entrySet()) {
	    int questionID 	= entry.getValue();
	    int ansID		= entry.getKey();
	    String title 	= db.getPostTitle(questionID);
	    if (title == null){
		//answer's question had bad data in it - extract title manually
		for (int i = 0 ; i < ansID ; i++){
		    Element e = (Element) nodes.item(i);
		    int postID = -1;
		    try{
			postID = Integer.parseInt(e.getAttribute(PostAttributes.ID.value()));
		    } catch (NumberFormatException e1) {continue;}
		    if (postID == questionID){
			title = e.getAttribute(PostAttributes.TITLE.value());
			break;
		    }
		}
	    }
	    if (title == null){
		title = "Unknown Title";
	    }
	    db.updatePostTitle(title, ansID);
	}
	
    }

    /**
     * 
     * Set all posts in answers to be accepted answers
     * 
     * @param answers
     * @param db
     * @throws SQLException
     */
    private void addAcceptedAnswers(ArrayList<Integer> answers, DBhandler db) throws SQLException{
	System.out.println("- Setting accepted answers...");
	for (Integer postID : answers) {
	    db.setAcceptedAnswer(postID);
	}
    }
    
    /**
     * @param users - list of users that has posts
     * 
     * @throws SQLException 
     * @throws ClassNotFoundException 
     * @throws IllegalAccessException 
     * @throws InstantiationException 
     */
    private void loadUsers(Document doc, ArrayList<Integer> knownUsers)
    		throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException{
	DBhandler db = new DBhandler();
	NodeList nodes = doc.getElementsByTagName(ELEMENT_DATA_NAME);
	System.out.println("-- Loading " + knownUsers.size() + " users...");
	//iterate nodes and extract code object from each one
	for (int i = 0 ; i < nodes.getLength() ; i++){
	    int userID, reputation;
	    String displayName;
	    //get element that contains our data as attributes
	    Element e = (Element) nodes.item(i);
	    //get attributes
	    try{
		//mandatory attribute
		userID		= Integer.parseInt(e.getAttribute(UserAttributes.ID.value()));
		try{
		    reputation	= Integer.parseInt(e.getAttribute(UserAttributes.REPUTATION.value()));
		} catch (NumberFormatException e1) {
		    reputation = -1;
		}
		displayName	= e.getAttribute(UserAttributes.DISPLAY_NAME.value());
	    } catch (NumberFormatException  e1) {
		//failed to parse integer - continue to next user
		continue;
	    }
	    //varity that the user is a known users (has at least one post entry in the DB)
	    if (!knownUsers.contains(userID)){
		//user is unknown - disregard him.
		continue;
	    }
	    //add user to DB.
	    db.addUser(userID, displayName, reputation);
	}
	//if there are elements left waiting for a commit
	db.executeBatches();
    }
    
    /**
     * @param posts - list of posts id's which votes will be loaded
     * @param doc
     * @throws SQLException 
     * @throws ClassNotFoundException 
     * @throws IllegalAccessException 
     * @throws InstantiationException 
     */
    private void loadVotes(Document doc, ArrayList<Integer> posts) 
    		throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException{
	DBhandler db = new DBhandler();
	PostVotesContainer votes = new PostVotesContainer(); 
	NodeList nodes = doc.getElementsByTagName(ELEMENT_DATA_NAME);
	System.out.println("-- Found " + nodes.getLength() + " votes. NOTE: irelevant votes will not be loaded...");
	//iterate nodes and extract code object from each one
	for (int i = 0 ; i < nodes.getLength() ; i++){
	    int postID, voteType;
	    Element e = (Element) nodes.item(i);
	    try{
		postID 		= Integer.parseInt(e.getAttribute(VotesAttributes.POST_ID.value()));
		voteType	= Integer.parseInt(e.getAttribute(VotesAttributes.VOTE_TYPE.value()));
	    } catch (NumberFormatException e1) {
		//failed to parse integer - continue to next user
		continue;
	    }
	    //check that vote type is up/dwon only
	    if (!VotesAttributes.VOTE_TYPE.isKnownVoteType(voteType)){
		//type unknown - disregard it
		continue;
	    }
	    //varify that vote is referencing a known post
	    if (!posts.contains(postID)){
		//unkown post - disregard it
		continue;
	    }

	    //update votes conteiner
	    if (voteType == VotesAttributes.UP_VOTE){
		votes.incUp(postID);
	    } else {
		votes.incDown(postID);
	    }
	}
	System.out.println("-- Adding " + votes.getAllVotes().size() + " vote objects to DB");
	//Finished votes extraction - now update posts with the votes.
	for (Votes v: votes.getAllVotes()){
	    db.addVotes(v.getPostID(), v.getUp(), v.getDown());
	}
	//if there are elements left waiting for a commit
	db.executeBatches();
    }
    
    /**
     * Load data from XML to MySQL DB
     * 
     * @param freshDB - if true any existing tables will be deleted, and new one will be created
     * @param loadPosts
     * @param loadUsers
     * @param loadVotes
     * @param loadOnlyJava - if set, only posts containing Java tag will be loaded
     * @throws SQLException
     * @throws InstantiationException
     * @throws IllegalAccessException
     * @throws ClassNotFoundException
     * @throws SAXException
     * @throws IOException
     * @throws ParserConfigurationException
     */
    public void loadDataDump(boolean freshDB, boolean loadPosts, boolean loadUsers, boolean loadVotes, boolean loadOnlyJava) 
    		throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException, 
    		SAXException, IOException, ParserConfigurationException{
	DBhandler db = new DBhandler();
	if (freshDB){
	    //create a fresh Database
	    System.out.println("Dumping old DB...");
	    db.dropDB();
	    System.out.println("Creating new DB...");
	    db.createTables();
	}
	
	//first - load posts
	if (loadPosts){
	    System.err.println("+ Clearing posts table...");
	    db.clearTable(DBhandler.DB.TableNames.POSTS);
	    System.out.println("Loading posts...");
	    loadPosts(getXML(postsXML), 0, -1, loadOnlyJava);
	}
	//load users
	if (loadUsers){
	    System.err.println("+ Clearing users table...");
	    db.clearTable(DBhandler.DB.TableNames.USERS);
	    System.out.println("Loading users...");
	    loadUsers(getXML(usersXML), db.fetchIDs(DBhandler.DB.Posts.USER_ID.value(), DBhandler.DB.TableNames.POSTS));
	}
	//load votes
	if (loadVotes){
	    System.out.println("Loading votes");
	    loadVotes(getXML(votesXML), db.fetchIDs(DBhandler.DB.Posts.ID.value(), DBhandler.DB.TableNames.POSTS));
	}
	
	System.out.println("Data dump loaded!");
    }
    
    /**
     * Check if {tag} exists in {source} string
     * 
     * @param source
     * @param tag
     * @return true iff source contains tag
     */
    private boolean tagExists(String source, String tag){
	return source.contains(tag);
    }        
}
